<!DOCTYPE html>

<html xmlns="http://www.w3.org/1999/xhtml">

<head>

<meta charset="utf-8" />
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="generator" content="pandoc" />


<meta name="author" content="Wenqiang Feng &amp; Ming Chen" />


<title>Decision tree Classification</title>

<script src="site_libs/jquery-1.11.3/jquery.min.js"></script>
<meta name="viewport" content="width=device-width, initial-scale=1" />
<link href="site_libs/bootstrap-3.3.5/css/cosmo.min.css" rel="stylesheet" />
<script src="site_libs/bootstrap-3.3.5/js/bootstrap.min.js"></script>
<script src="site_libs/bootstrap-3.3.5/shim/html5shiv.min.js"></script>
<script src="site_libs/bootstrap-3.3.5/shim/respond.min.js"></script>
<script src="site_libs/navigation-1.1/tabsets.js"></script>
<link href="site_libs/highlightjs-1.1/textmate.css" rel="stylesheet" />
<script src="site_libs/highlightjs-1.1/highlight.js"></script>
<link href="site_libs/font-awesome-4.5.0/css/font-awesome.min.css" rel="stylesheet" />

<style type="text/css">code{white-space: pre;}</style>
<style type="text/css">
  pre:not([class]) {
    background-color: white;
  }
</style>
<script type="text/javascript">
if (window.hljs && document.readyState && document.readyState === "complete") {
   window.setTimeout(function() {
      hljs.initHighlighting();
   }, 0);
}
</script>



<style type="text/css">
h1 {
  font-size: 34px;
}
h1.title {
  font-size: 38px;
}
h2 {
  font-size: 30px;
}
h3 {
  font-size: 24px;
}
h4 {
  font-size: 18px;
}
h5 {
  font-size: 16px;
}
h6 {
  font-size: 12px;
}
.table th:not([align]) {
  text-align: left;
}
</style>


</head>

<body>

<style type = "text/css">
.main-container {
  max-width: 940px;
  margin-left: auto;
  margin-right: auto;
}
code {
  color: inherit;
  background-color: rgba(0, 0, 0, 0.04);
}
img {
  max-width:100%;
  height: auto;
}
.tabbed-pane {
  padding-top: 12px;
}
button.code-folding-btn:focus {
  outline: none;
}
</style>


<style type="text/css">
/* padding for bootstrap navbar */
body {
  padding-top: 51px;
  padding-bottom: 40px;
}
/* offset scroll position for anchor links (for fixed navbar)  */
.section h1 {
  padding-top: 56px;
  margin-top: -56px;
}

.section h2 {
  padding-top: 56px;
  margin-top: -56px;
}
.section h3 {
  padding-top: 56px;
  margin-top: -56px;
}
.section h4 {
  padding-top: 56px;
  margin-top: -56px;
}
.section h5 {
  padding-top: 56px;
  margin-top: -56px;
}
.section h6 {
  padding-top: 56px;
  margin-top: -56px;
}
</style>

<script>
// manage active state of menu based on current page
$(document).ready(function () {
  // active menu anchor
  href = window.location.pathname
  href = href.substr(href.lastIndexOf('/') + 1)
  if (href === "")
    href = "index.html";
  var menuAnchor = $('a[href="' + href + '"]');

  // mark it active
  menuAnchor.parent().addClass('active');

  // if it's got a parent navbar menu mark it active as well
  menuAnchor.closest('li.dropdown').addClass('active');
});
</script>


<div class="container-fluid main-container">

<!-- tabsets -->
<script>
$(document).ready(function () {
  window.buildTabsets("TOC");
});
</script>

<!-- code folding -->






<div class="navbar navbar-inverse  navbar-fixed-top" role="navigation">
  <div class="container">
    <div class="navbar-header">
      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar">
        <span class="icon-bar"></span>
        <span class="icon-bar"></span>
        <span class="icon-bar"></span>
      </button>
      <a class="navbar-brand" href="index.html">Learning Apache Spark</a>
    </div>
    <div id="navbar" class="navbar-collapse collapse">
      <ul class="nav navbar-nav">
        <li>
  <a href="index.html">
    <span class="fa fa-home"></span>
     
    Home
  </a>
</li>
<li class="dropdown">
  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
    <span class="fa fa-gear"></span>
     
    Getting Start
     
    <span class="caret"></span>
  </a>
  <ul class="dropdown-menu" role="menu">
    <li class="divider"></li>
    <li class="dropdown-header">Installations</li>
    <li>
      <a href="install.html">Installations</a>
    </li>
    <li class="divider"></li>
    <li class="dropdown-header">Integrate Spark</li>
    <li>
      <a href="pyspark-on-rodeo.html">Integrate spark with Rodeo</a>
    </li>
    <li>
      <a href="pyspark-on-jupyter.html">Integrate spark with Jupyter</a>
    </li>
    <li>
      <a href="spark-on-jetstream-cloud.html">Spark on jetstream cloud</a>
    </li>
  </ul>
</li>
<li class="dropdown">
  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
    <span class="fa fa-gear"></span>
     
    Data Preparation
     
    <span class="caret"></span>
  </a>
  <ul class="dropdown-menu" role="menu">
    <li>
      <a href="01_entry_points_to_spark.html">Entry points to spark</a>
    </li>
    <li>
      <a href="02_rdd_object.html">RDD object</a>
    </li>
    <li>
      <a href="03_dataframe_object.html">DataFrame object</a>
    </li>
    <li>
      <a href="categorical-data.html">Categorial data</a>
    </li>
    <li>
      <a href="HashingTF-and-CountVectorizer.html">HashingTF and CountVectorizer</a>
    </li>
  </ul>
</li>
<li class="dropdown">
  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
    <span class="fa fa-gear"></span>
     
    Machine Learning
     
    <span class="caret"></span>
  </a>
  <ul class="dropdown-menu" role="menu">
    <li class="divider"></li>
    <li class="dropdown-header">Regression</li>
    <li>
      <a href="linear-regression.html">Linear regression</a>
    </li>
    <li>
      <a href="generalized-linear-regression.html">Generalized linear regression</a>
    </li>
    <li>
      <a href="dttreeR.html">Decision tree Regression</a>
    </li>
    <li>
      <a href="randomforest.html">Random Forest Regression</a>
    </li>
    <li class="divider"></li>
    <li class="dropdown-header">Classification</li>
    <li>
      <a href="kmeans.html">Kmeans Classification</a>
    </li>
    <li>
      <a href="dttreeC.html">Decision tree Classification</a>
    </li>
    <li>
      <a href="randomforestC.html">Random Forest Classification</a>
    </li>
    <li>
      <a href="logistic-regression.html">Logistic Regression</a>
    </li>
    <li>
      <a href="svm.html">Support Vector Machine</a>
    </li>
    <li>
      <a href="naive-baye.html">Naive Bayes</a>
    </li>
    <li class="divider"></li>
    <li class="dropdown-header">Neural Network</li>
    <li>
      <a href="fnn.html">Feedforward Neural Network</a>
    </li>
    <li class="divider"></li>
    <li class="dropdown-header">Natural Language Processing</li>
    <li>
      <a href="nlpC.html">Text Classification</a>
    </li>
    <li>
      <a href="nlpLDA.html">Topic Model LDA</a>
    </li>
    <li class="divider"></li>
    <li class="dropdown-header">Social Network Analysis</li>
    <li>
      <a href="sna.html">Social Network Analysis</a>
    </li>
  </ul>
</li>
<li class="dropdown">
  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
    <span class="fa fa-gear"></span>
     
    Module Tuning and Evaluation
     
    <span class="caret"></span>
  </a>
  <ul class="dropdown-menu" role="menu">
    <li class="divider"></li>
    <li class="dropdown-header">Module Tuning</li>
    <li>
      <a href="regularization.html">Regularization</a>
    </li>
    <li>
      <a href="k-folds-cross-validation.html">K-folds Cross Validation</a>
    </li>
  </ul>
</li>
      </ul>
      <ul class="nav navbar-nav navbar-right">
        <li>
  <a href="https://github.com/MingChen0919/learning-apache-spark">
    <span class="fa fa-github"></span>
     
  </a>
</li>
<li>
  <a href="https://twitter.com/mingchen0919">
    <span class="fa fa-twitter"></span>
     
  </a>
</li>
      </ul>
    </div><!--/.nav-collapse -->
  </div><!--/.container -->
</div><!--/.navbar -->

<div class="fluid-row" id="header">



<h1 class="title toc-ignore">Decision tree Classification</h1>
<h4 class="author"><em>Wenqiang Feng &amp; Ming Chen</em></h4>
<h4 class="date"><em>2/17/2017</em></h4>

</div>


<div id="remark" class="section level3">
<h3>Remark:</h3>
<ul>
<li><p>You can download the complete <a href="./ipynb/DecisionTreeC3.ipynb">ipython notebook (3 classes)</a> and <a href="./ipynb/DecisionTreeC7.ipynb">ipython notebook (7 classes)</a> for the this session.</p></li>
<li><p>More details can be found on the offical website for <a href="https://spark.apache.org/docs/latest/ml-classification-regression.html">pyspark.ml package</a></p></li>
</ul>
<p><a href="https://en.wikipedia.org/wiki/Decision_tree">Wikipedia</a>: A decision tree is a decision support tool that uses a tree-like graph or model of decisions and their possible consequences, including chance event outcomes, resource costs, and utility. It is one way to display an algorithm.</p>
<p>Decision trees are commonly used in operations research, specifically in decision analysis, to help identify a strategy most likely to reach a goal, but are also a popular tool in machine learning.</p>
<p>Decision tree learning is a method commonly used in data mining.[1] The goal is to create a model that predicts the value of a target variable based on several input variables.It can be used to do the regression and classfication.</p>
</div>
<div id="set-up-spark-context-and-sparksession" class="section level3">
<h3>1. Set up spark context and SparkSession</h3>
<pre class="python"><code>from pyspark.sql import SparkSession

spark = SparkSession \
    .builder \
    .appName(&quot;Python Spark Decision tree Classification&quot;) \
    .config(&quot;spark.some.config.option&quot;, &quot;some-value&quot;) \
    .getOrCreate()</code></pre>
</div>
<div id="load-dataset" class="section level3">
<h3>2. Load dataset</h3>
<pre class="python"><code>df = spark.read.format(&#39;com.databricks.spark.csv&#39;).\
                               options(header=&#39;true&#39;, \
                               inferschema=&#39;true&#39;).\
                 load(&quot;./data/WineData.csv&quot;,header=True);</code></pre>
<ul>
<li>define UDF (User Defined Function)</li>
</ul>
<pre class="python"><code># Convert to float format
def string_to_float(x):
    return float(x)

# 
def condition(r):
    if (0&lt;= r &lt;= 4):
        label = &quot;low&quot; 
    elif(4&lt; r &lt;= 6):
        label = &quot;medium&quot;
    else: 
        label = &quot;high&quot; 
    return label</code></pre>
<ul>
<li>reqired library</li>
</ul>
<pre class="python"><code>from pyspark.sql.functions import udf
from pyspark.sql.types import StringType, DoubleType
string_to_float_udf = udf(string_to_float, DoubleType())
quality_udf = udf(lambda x: condition(x), StringType())</code></pre>
<ul>
<li>convert to 3 classes</li>
</ul>
<pre class="python"><code>df = df.withColumn(&quot;quality&quot;, quality_udf(&quot;quality&quot;))</code></pre>
<ul>
<li>check the schema</li>
</ul>
<pre class="python"><code>df.printSchema()</code></pre>
<pre class="python"><code># output 
root
 |-- fixed acidity: double (nullable = true)
 |-- volatile acidity: double (nullable = true)
 |-- citric acid: double (nullable = true)
 |-- residual sugar: double (nullable = true)
 |-- chlorides: double (nullable = true)
 |-- free sulfur dioxide: double (nullable = true)
 |-- total sulfur dioxide: double (nullable = true)
 |-- density: double (nullable = true)
 |-- pH: double (nullable = true)
 |-- sulphates: double (nullable = true)
 |-- alcohol: double (nullable = true)
 |-- quality: string (nullable = true)
</code></pre>
<ul>
<li>preview the dataset</li>
</ul>
<pre class="python"><code>df.show(4)</code></pre>
<pre class="python"><code>#output 
+-------------+----------------+-----------+--------------+---------+-------------------+--------------------+-------+----+---------+-------+-------+
|fixed acidity|volatile acidity|citric acid|residual sugar|chlorides|free sulfur dioxide|total sulfur dioxide|density|  pH|sulphates|alcohol|quality|
+-------------+----------------+-----------+--------------+---------+-------------------+--------------------+-------+----+---------+-------+-------+
|          7.4|             0.7|        0.0|           1.9|    0.076|               11.0|                34.0| 0.9978|3.51|     0.56|    9.4|   high|
|          7.8|            0.88|        0.0|           2.6|    0.098|               25.0|                67.0| 0.9968| 3.2|     0.68|    9.8|   high|
|          7.8|            0.76|       0.04|           2.3|    0.092|               15.0|                54.0|  0.997|3.26|     0.65|    9.8|   high|
|         11.2|            0.28|       0.56|           1.9|    0.075|               17.0|                60.0|  0.998|3.16|     0.58|    9.8|   high|
+-------------+----------------+-----------+--------------+---------+-------------------+--------------------+-------+----+---------+-------+-------+
only showing top 4 rows</code></pre>
<ul>
<li>load the required library for the model</li>
</ul>
<pre class="python"><code>from pyspark.ml.linalg import Vectors # !!!!caution: not from pyspark.mllib.linalg import Vectors
from pyspark.ml import Pipeline
from pyspark.ml.feature import IndexToString,StringIndexer, VectorIndexer
from pyspark.ml.classification import DecisionTreeClassifier
from pyspark.ml.tuning import CrossValidator, ParamGridBuilder
from pyspark.ml.evaluation import MulticlassClassificationEvaluator</code></pre>
<ul>
<li>convert data to PySpark ML data framework</li>
</ul>
<pre class="python"><code>def transData(data):
    return data.rdd.map(lambda r: [Vectors.dense(r[:-1]),r[-1]]).toDF([&#39;features&#39;,&#39;label&#39;])</code></pre>
<pre class="python"><code>data = transData(df)</code></pre>
<ul>
<li>check data framework</li>
</ul>
<pre class="python"><code>data.show(3)</code></pre>
<pre class="python"><code># output 
+--------------------+-----+
|            features|label|
+--------------------+-----+
|[7.4,0.7,0.0,1.9,...| high|
|[7.8,0.88,0.0,2.6...| high|
|[7.8,0.76,0.04,2....| high|
+--------------------+-----+
only showing top 3 rows</code></pre>
<ul>
<li>Index labels, adding metadata to the label column</li>
</ul>
<pre class="python"><code># Index labels, adding metadata to the label column
labelIndexer = StringIndexer(inputCol=&#39;label&#39;,
                             outputCol=&#39;indexedLabel&#39;).fit(data)
labelIndexer.transform(data).show(6)</code></pre>
<pre class="python"><code># output 
+--------------------+-----+------------+
|            features|label|indexedLabel|
+--------------------+-----+------------+
|[7.4,0.7,0.0,1.9,...| high|         0.0|
|[7.8,0.88,0.0,2.6...| high|         0.0|
|[7.8,0.76,0.04,2....| high|         0.0|
|[11.2,0.28,0.56,1...| high|         0.0|
|[7.4,0.7,0.0,1.9,...| high|         0.0|
|[7.4,0.66,0.0,1.8...| high|         0.0|
+--------------------+-----+------------+
only showing top 6 rows</code></pre>
<ul>
<li>convert the features to ML data framework</li>
</ul>
<pre class="python"><code># Automatically identify categorical features, and index them.
# Set maxCategories so features with &gt; 4 distinct values are treated as continuous.
featureIndexer =VectorIndexer(inputCol=&quot;features&quot;, \
                              outputCol=&quot;indexedFeatures&quot;, \
                              maxCategories=4).fit(data)

featureIndexer.transform(data).show(6) </code></pre>
<pre class="python"><code>+--------------------+-----+--------------------+
|            features|label|     indexedFeatures|
+--------------------+-----+--------------------+
|[7.4,0.7,0.0,1.9,...| high|[7.4,0.7,0.0,1.9,...|
|[7.8,0.88,0.0,2.6...| high|[7.8,0.88,0.0,2.6...|
|[7.8,0.76,0.04,2....| high|[7.8,0.76,0.04,2....|
|[11.2,0.28,0.56,1...| high|[11.2,0.28,0.56,1...|
|[7.4,0.7,0.0,1.9,...| high|[7.4,0.7,0.0,1.9,...|
|[7.4,0.66,0.0,1.8...| high|[7.4,0.66,0.0,1.8...|
+--------------------+-----+--------------------+
only showing top 6 rows</code></pre>
</div>
<div id="train-a-decisiontree-model" class="section level3">
<h3>Train a DecisionTree model</h3>
<pre class="python"><code># Train a DecisionTree model
dTree = DecisionTreeClassifier(labelCol=&#39;indexedLabel&#39;, featuresCol=&#39;indexedFeatures&#39;)</code></pre>
<pre class="python"><code># Convert indexed labels back to original labels.
labelConverter = IndexToString(inputCol=&quot;prediction&quot;, outputCol=&quot;predictedLabel&quot;,
                               labels=labelIndexer.labels)</code></pre>
<pre class="python"><code># Chain indexers and tree in a Pipeline
pipeline = Pipeline(stages=[labelIndexer, featureIndexer, dTree,labelConverter])</code></pre>
<pre class="python"><code># Split the data into training and test sets (30% held out for testing)
(trainingData, testData) = data.randomSplit([0.6, 0.4])</code></pre>
<pre class="python"><code># Train model.  This also runs the indexers.
model = pipeline.fit(trainingData)</code></pre>
<pre class="python"><code># Make predictions.
predictions = model.transform(testData)</code></pre>
<pre class="python"><code># Select example rows to display.
predictions.select(&quot;features&quot;,&quot;label&quot;,&quot;predictedLabel&quot;).show(5)</code></pre>
<pre class="python"><code>#output
+--------------------+-----+--------------+
|            features|label|predictedLabel|
+--------------------+-----+--------------+
|[4.6,0.52,0.15,2....| high|          high|
|[4.9,0.42,0.0,2.1...| high|          high|
|[5.0,0.4,0.5,4.3,...| high|          high|
|[5.0,0.42,0.24,2....| high|          high|
|[5.0,1.02,0.04,1....| high|          high|
+--------------------+-----+--------------+
only showing top 5 rows</code></pre>
<pre class="python"><code># Select (prediction, true label) and compute test error
evaluator = MulticlassClassificationEvaluator(
    labelCol=&quot;indexedLabel&quot;, predictionCol=&quot;prediction&quot;, metricName=&quot;accuracy&quot;)
accuracy = evaluator.evaluate(predictions)
print(&quot;Predictions accuracy = %g, Test Error = %g&quot; % (accuracy,(1.0 - accuracy)))</code></pre>
</div>

<center>Copyright &copy; 2017 Ming Chen  & Wenqiang Feng. All rights reserved.</center>



</div>

<script>

// add bootstrap table styles to pandoc tables
function bootstrapStylePandocTables() {
  $('tr.header').parent('thead').parent('table').addClass('table table-condensed');
}
$(document).ready(function () {
  bootstrapStylePandocTables();
});


</script>

<!-- dynamically load mathjax for compatibility with self-contained -->
<script>
  (function () {
    var script = document.createElement("script");
    script.type = "text/javascript";
    script.src  = "https://mathjax.rstudio.com/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML";
    document.getElementsByTagName("head")[0].appendChild(script);
  })();
</script>

</body>
</html>
